package com.asiainfo.toutiao;

import cn.edu.hfut.dmic.webcollector.model.CrawlDatum;
import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
import cn.edu.hfut.dmic.webcollector.model.Page;
import cn.edu.hfut.dmic.webcollector.plugin.berkeley.BreadthCrawler;
import com.asiainfo.souhu.ZhouSouhu;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.sun.jmx.snmp.SnmpOidTable;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.util.TextUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sun.awt.AWTIcon32_java_icon16_png;

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * No1FinanceAndEconomics
 * 一号财经今日头条号
 *
 * @Author jhr
 * @Date 2022/3/30
 */
public class No1FinanceAndEconomics extends BreadthCrawler {
    /**
     * 构造一个基于伯克利DB的爬虫
     * 伯克利DB文件夹为crawlPath，crawlPath中维护了历史URL等信息
     * 不同任务不要使用相同的crawlPath
     * 两个使用相同crawlPath的爬虫并行爬取会产生错误
     *
     * @param crawlPath 伯克利DB使用的文件夹
     * @param autoParse 是否根据设置的正则自动探测新URL
     */

    private static final Logger logger = LoggerFactory.getLogger(No1FinanceAndEconomics.class);
    private static final String crawlPath = "./toutiao/zhou";
    // https://profile.zjurl.cn/api/feed_backflow/profile_share/v1/?app_name=news_article&category=profile_all&visited_uid=5464344913&stream_api_version=82&request_source=1&offset=0&user_id=5464344913&media_id=5878900007&appId=1286&appType=mobile_detail_web&isAndroid=true&isIOS=false&isMobile=true&cookie_enabled=true&screen_width=386&screen_height=561&browser_language=zh-CN&browser_platform=MacIntel&browser_name=firefox&browser_version=99.0.4844.51&browser_online=true    static String RootUrl = "";
    static String RootUrl = "";
    //    接口url：获取每篇文章的url,改变pNo后的数值，实现获取信息
    static String IUrl = "";
    //    某一个文章的url
//    https://m.toutiao.com/i7077818835247645223/info/v2/
//    static String seed = "";
    //    匹配文章信息的URL
//    https://m.toutiao.com/i7077818835247645223/info/v2/
//    https://m.toutiao.com/i.*/info/v2/
    static String RegularUrl = "";

    public No1FinanceAndEconomics(String Seed, String regularUrl) {
        super(crawlPath, false);
        IUrl = Seed;
        RegularUrl = regularUrl;
//        crawlPath = CrawlPath;
        CrawlDatum crawlDatum = new CrawlDatum(IUrl).meta("depth", "2");
        addSeed(crawlDatum);
        this.addRegex(RegularUrl);
        setThreads(1);
//      是否进行断点爬取，默认为 false
//        setResumable(true);

    }

    @Override
    public void visit(Page page, CrawlDatums next) {
        if (page.matchUrl(RegularUrl)) {
//           获取符合的url，进行json解析，拿到想要的数据
            System.out.println("文章json数据url:" + page.url());
            JsonElement data = getLink(page.url());
            System.out.println("获取到接口数据data:" + data);
//            判断数据接口的内容，有文章，视频，动态等类型，需要分类判断
            if (data.getAsJsonObject().get("title") != null) {
                System.out.println("=====发布的为文章或者视频形式=====");
//            标题
                String title = String.valueOf(data.getAsJsonObject().get("title")).replaceAll("\"", "");
                System.out.println("标题:" + title);
//            发布时间
                String publishTime = String.valueOf(data.getAsJsonObject().get("publish_time")).replaceAll("\"", "");
                SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
//            long finaWay = sdf.parse(sdf.format(new Date())).getTime();

//            String date = TimeStamp2Date(publishTime, "yyyy-MM-dd HH:mm:ss");
                System.out.println(Long.valueOf(publishTime));
                String format = sdf.format(new Date(Long.parseLong(publishTime) * 1000));
                try {
                    System.out.println("发布时间:" + sdf.parse(format));
                } catch (ParseException e) {
                    e.printStackTrace();
                }
//            正文
                String content = String.valueOf(data.getAsJsonObject().get("content"));
                System.out.println("正文:" + content.replaceAll("\"", "").replaceAll("<[^>]*>", "").replaceAll(" ", "").replaceAll("\r|\n", ""));
//          文章链接
                String url = String.valueOf(data.getAsJsonObject().get("url"));
                System.out.println("文章url:" + url.replaceAll("\"", ""));
//            阅读量
                String impressionCount = String.valueOf(data.getAsJsonObject().get("impression_count")).replaceAll("\"", "");
                System.out.println("阅读量:" + Integer.parseInt(impressionCount));
//           作者
                String detailSource = String.valueOf(data.getAsJsonObject().get("detail_source"));
                System.out.println("作者:" + detailSource.replaceAll("\"", ""));
//            来源：今日头条
                String source = "今日头条";
                System.out.println("来源的网站类型：" + source);

            } else if (data.getAsJsonObject().get("thread") != null) {
                System.out.println("====发布类型的为动态：====");
//                获取thread_base
                JsonElement threadBaseElement = data.getAsJsonObject().get("thread").getAsJsonObject().get("thread_base");
//            标题,字符处理，去除空格，换行符等
                String threadTitle = String.valueOf(threadBaseElement.getAsJsonObject().get("title")).replaceAll("\"", "").replaceAll("<[^>]*>", "").replaceAll(" ", "").trim().replaceAll("[\r\n]", "").replaceAll("\n", "").replaceAll("\\\\s*|\\t|\\r|\\n*", "").replaceAll("nn", "");
                System.out.println("标题:" + threadTitle);
//            发布时间
                String publishTime = String.valueOf(threadBaseElement.getAsJsonObject().get("create_time")).replaceAll("\"", "");
                SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
//            long finaWay = sdf.parse(sdf.format(new Date())).getTime();
//            String date = TimeStamp2Date(publishTime, "yyyy-MM-dd HH:mm:ss");
//                System.out.println(Long.valueOf(publishTime));
                String format = sdf.format(new Date(Long.parseLong(publishTime) * 1000));
                try {
                    System.out.println("发布时间:" + sdf.parse(format));
                } catch (ParseException e) {
                    e.printStackTrace();
                }
//            正文
                String content = String.valueOf(threadBaseElement.getAsJsonObject().get("content"));
                content = content.replaceAll("\"", "").replaceAll("<[^>]*>", "").replaceAll(" ", "").trim().replaceAll("[\r\n]", "").replaceAll("\n", "").replaceAll("\\\\s*|\\t|\\r|\\n*", "").replaceAll("nn", "");
                System.out.println("正文:" +content);
//          文章链接
                String share_url = String.valueOf(threadBaseElement.getAsJsonObject().get("share_url"));
                System.out.println("文章url:" + share_url.replaceAll("\"", ""));
//            阅读量
                String read_count = String.valueOf(threadBaseElement.getAsJsonObject().get("action").getAsJsonObject().get("read_count")).replaceAll("\"", "").replaceAll("\r|\n", "");
                System.out.println("阅读量:" + Integer.parseInt(read_count));
//           作者
                String detailSource = String.valueOf(threadBaseElement.getAsJsonObject().get("user").getAsJsonObject().get("info").getAsJsonObject().get("name"));
                System.out.println("作者:" + detailSource.replaceAll("\"", ""));
//            来源：今日头条
                String source = "今日头条";
                System.out.println("来源的网站类型：" + source);
            } else {
                System.out.println("获取数据失败");
            }

        } else {
            System.out.println("种子url：" + page.url());
//            拿到每个文章的id
            JsonElement articleIdJson = getLink(page.url());
            for (JsonElement element : articleIdJson.getAsJsonArray()) {
                System.out.println(element);
                JsonElement content = element.getAsJsonObject().get("content");
                System.out.println(content.toString());
//                对content内容处理:去除特殊字符
                String regEx = "[\n`~!@#$%^&*()+=|{}':;',\\[\\].<>/?~！@#￥%……&*()——+|{}【】‘；：”“’。， 、？]";
                Pattern p = Pattern.compile(regEx);
                Matcher m = p.matcher(content.toString());//这里把想要替换的字符串传进来
                String newString = m.replaceAll("").trim();//这里是将特殊字符换为空字符串,""代表直接去掉
                System.out.println(newString);
//                去除特殊字符后，查找文章id
                System.out.println(newString.indexOf("group_id_str"));
//                查找替换
//                String[] articleIds = StringUtils.substringsBetween(newString, "group_id_str", "\\");
                String[] articleIds = StringUtils.substringsBetween(newString, "group_id_str\\\"\\\"", "\\\"");
//                查找不到id，则说明为回答问题类型数据,那么便根据logpb_group_id查询id
                if (articleIds == null) {
                    articleIds = StringUtils.substringsBetween(newString, "logpb_group_id\\\"\\\"", "\\\"");
                    System.out.println("id为回答类型");
                }
//                for (String articleId : articleIds) {
//                    System.out.println(articleId);
//                }

                String regRx = RegularUrl;
//                匹配符合正则的网站
                String link = "https://m.toutiao.com/i" + articleIds[0] + "/info/v2/";
                if (link.matches(regRx)) {
                    CrawlDatum meta = new CrawlDatum(link).meta("depth", "1").meta("refer", link);
                    next.add(meta);
                } else {
                    System.out.println("正则URL不匹配！！！");
                }
            }
        }

        return ;
    }

    //  获取接口url中的具体文章url
    private JsonElement getLink(String s) {
//        System.out.println(s+1);
        String jsonStr = getHtmlByUrl(s);
        JsonParser parse = new JsonParser();  //创建json解析器
        JsonObject jsonObject = (JsonObject) parse.parse(jsonStr);
        //                .getAsJsonObject().get("pcArticleVOS");
//带有link的数组，一次20个
//        获取data数据
        return jsonObject.get("data");
    }

    //   接口解析
    public String getHtmlByUrl(String url) {
        String text = null;
        int code = 0;
        try {
            while (code != 200) {
                StringBuffer html = new StringBuffer();
                URL Url = new URL(url);
                HttpURLConnection conn = (HttpURLConnection) Url.openConnection();
                //这个很必要，否则就是403
                conn.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)");
                code = conn.getResponseCode();
                InputStreamReader isr = new InputStreamReader(conn.getInputStream());
                BufferedReader br = new BufferedReader(isr);
                String temp;
                while ((temp = br.readLine()) != null) {
                    html.append(temp).append("\n");
                }
                br.close();
                isr.close();
                text = html.toString();
            }
        } catch (Exception e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        return text;
    }

    /**
     * Java将Unix时间戳转换成指定格式日期字符串
     *
     * @param timestampString 时间戳 如："1473048265";
     * @param formats         要格式化的格式 默认："yyyy-MM-dd HH:mm:ss";
     * @return 返回结果 如："2016-09-05 16:06:42";
     */
    public String TimeStamp2Date(String timestampString, String formats) {
        if (TextUtils.isEmpty(formats))
            formats = "yyyy-MM-dd HH:mm:ss";
        long timestamp = Long.parseLong(timestampString) * 1000;
        return new SimpleDateFormat(formats, Locale.CHINA).format(new Date(timestamp));
    }


    public static void main(String[] args) throws Exception {
        String regular = "https://m.toutiao.com/i.*/info/v2/";
        String seed = "https://profile.zjurl.cn/api/feed_backflow/profile_share/v10/?app_name=news_article&category=profile_all&visited_uid=5464344913&stream_api_version=82&request_source=1&offset=0&user_id=5464344913&media_id=5878900007&appId=1286&appType=mobile_detail_web&isAndroid=true&isIOS=false&isMobile=true&cookie_enabled=true&screen_width=387&screen_height=561&browser_language=zh-CN&browser_platform=MacIntel&browser_name=firefox&browser_version=99.0.4844.51&browser_online=true";
//        绝对军视，接口
//        String seed2 = "https://profile.zjurl.cn/api/feed_backflow/profile_share/v1/?app_name=news_article&category=profile_all&visited_uid=50098859376&stream_api_version=82&request_source=1&offset=0&user_id=50098859376&media_id=50228741523&appId=1286&appType=mobile_detail_web&isAndroid=true&isIOS=false&isMobile=true&cookie_enabled=true&screen_width=760&screen_height=561&browser_language=zh-CN&browser_platform=MacIntel&browser_name=firefox&browser_version=99.0.4844.51&browser_online=true";
//        String seed2 = "https://profile.zjurl.cn/api/feed_backflow/profile_share/v1/?app_name=news_article&category=profile_all&visited_uid=2123879729075335&stream_api_version=82&request_source=1&offset=0&user_id=2123879729075335&media_id=1650872444039172&appId=1286&appType=mobile_detail_web&isAndroid=true&isIOS=false&isMobile=true&cookie_enabled=true&screen_width=760&screen_height=561&browser_language=zh-CN&browser_platform=MacIntel&browser_name=firefox&browser_version=99.0.4844.51&browser_online=true&";
        //        接口：url，获取每篇文章的url
        No1FinanceAndEconomics crawler = new No1FinanceAndEconomics(seed, regular);
        crawler.start(3);
    }
}
